

<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<head>
  <meta charset="utf-8" />
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
  
  <title>mindspore.nn.layer.activation &mdash; MindSpore master documentation</title>
  

  
  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />

  
  

  
  

  

  
  <!--[if lt IE 9]>
    <script src="../../../../_static/js/html5shiv.min.js"></script>
  <![endif]-->
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
        <script src="../../../../_static/jquery.js"></script>
        <script src="../../../../_static/underscore.js"></script>
        <script src="../../../../_static/doctools.js"></script>
        <script src="../../../../_static/language_data.js"></script>
        <script async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
    
    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>

    
    <link rel="index" title="Index" href="../../../../genindex.html" />
    <link rel="search" title="Search" href="../../../../search.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../../../../index.html" class="icon icon-home"> MindSpore
          

          
          </a>

          
            
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">MindSpore Python API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.html">mindspore</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.common.initializer.html">mindspore.common.initializer</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.communication.html">mindspore.communication</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.compression.html">mindspore.compression</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.context.html">mindspore.context</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.html">mindspore.dataset</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.audio.html">mindspore.dataset.audio</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.config.html">mindspore.dataset.config</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.text.html">mindspore.dataset.text</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.transforms.html">mindspore.dataset.transforms</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.vision.html">mindspore.dataset.vision</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.mindrecord.html">mindspore.mindrecord</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.nn.html">mindspore.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.nn.probability.html">mindspore.nn.probability</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.nn.transformer.html">mindspore.nn.transformer</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.numpy.html">mindspore.numpy</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.ops.html">mindspore.ops</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.parallel.html">mindspore.parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.parallel.nn.html">mindspore.parallel.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.profiler.html">mindspore.profiler</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.scipy.html">mindspore.scipy</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.train.html">mindspore.train</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.boost.html">mindspore.boost</a></li>
</ul>
<p class="caption"><span class="caption-text">MindSpore C++ API</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://www.mindspore.cn/lite/api/zh-CN/master/api_cpp/mindspore.html">MindSpore Lite↗</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../../../../index.html">MindSpore</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          

















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../../../../index.html" class="icon icon-home"></a> &raquo;</li>
        
          <li><a href="../../../index.html">Module code</a> &raquo;</li>
        
      <li>mindspore.nn.layer.activation</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <h1>Source code for mindspore.nn.layer.activation</h1><div class="highlight"><pre>
<span></span><span class="c1"># Copyright 2020-2021 Huawei Technologies Co., Ltd</span>
<span class="c1">#</span>
<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
<span class="c1"># you may not use this file except in compliance with the License.</span>
<span class="c1"># You may obtain a copy of the License at</span>
<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="c1"># See the License for the specific language governing permissions and</span>
<span class="c1"># limitations under the License.</span>
<span class="c1"># ============================================================================</span>
<span class="sd">&quot;&quot;&quot;activation&quot;&quot;&quot;</span>
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>

<span class="kn">from</span> <span class="nn">mindspore._checkparam</span> <span class="kn">import</span> <span class="n">Validator</span> <span class="k">as</span> <span class="n">validator</span>
<span class="kn">from</span> <span class="nn">mindspore._extends</span> <span class="kn">import</span> <span class="n">cell_attr_register</span>
<span class="kn">from</span> <span class="nn">mindspore.common</span> <span class="kn">import</span> <span class="n">dtype</span> <span class="k">as</span> <span class="n">mstype</span>
<span class="kn">from</span> <span class="nn">mindspore.common.parameter</span> <span class="kn">import</span> <span class="n">Parameter</span>
<span class="kn">from</span> <span class="nn">mindspore.common.tensor</span> <span class="kn">import</span> <span class="n">Tensor</span>
<span class="kn">from</span> <span class="nn">mindspore.ops</span> <span class="kn">import</span> <span class="n">functional</span> <span class="k">as</span> <span class="n">F</span>
<span class="kn">from</span> <span class="nn">mindspore.ops</span> <span class="kn">import</span> <span class="n">operations</span> <span class="k">as</span> <span class="n">P</span>
<span class="kn">from</span> <span class="nn">..cell</span> <span class="kn">import</span> <span class="n">Cell</span>

<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;Softmax&#39;</span><span class="p">,</span>
           <span class="s1">&#39;LogSoftmax&#39;</span><span class="p">,</span>
           <span class="s1">&#39;ReLU&#39;</span><span class="p">,</span>
           <span class="s1">&#39;ReLU6&#39;</span><span class="p">,</span>
           <span class="s1">&#39;Tanh&#39;</span><span class="p">,</span>
           <span class="s1">&#39;GELU&#39;</span><span class="p">,</span>
           <span class="s1">&#39;FastGelu&#39;</span><span class="p">,</span>
           <span class="s1">&#39;Sigmoid&#39;</span><span class="p">,</span>
           <span class="s1">&#39;PReLU&#39;</span><span class="p">,</span>
           <span class="s1">&#39;get_activation&#39;</span><span class="p">,</span>
           <span class="s1">&#39;LeakyReLU&#39;</span><span class="p">,</span>
           <span class="s1">&#39;HSigmoid&#39;</span><span class="p">,</span>
           <span class="s1">&#39;HSwish&#39;</span><span class="p">,</span>
           <span class="s1">&#39;ELU&#39;</span><span class="p">,</span>
           <span class="s1">&#39;LogSigmoid&#39;</span><span class="p">,</span>
           <span class="s1">&#39;SoftShrink&#39;</span><span class="p">,</span>
           <span class="s1">&#39;HShrink&#39;</span><span class="p">,</span>
           <span class="s1">&#39;CELU&#39;</span><span class="p">,</span>
           <span class="p">]</span>


<div class="viewcode-block" id="CELU"><a class="viewcode-back" href="../../../../api_python/nn/mindspore.nn.CELU.html#mindspore.nn.CELU">[docs]</a><span class="k">class</span> <span class="nc">CELU</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Continuously differentiable exponential linear units activation function.</span>

<span class="sd">    Applies the continuously differentiable exponential linear units function element-wise.</span>

<span class="sd">    .. math::</span>

<span class="sd">        \text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1))</span>

<span class="sd">    It returns element-wise :math:`\max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1))`.</span>

<span class="sd">    The picture about CELU looks like this `CELU &lt;https://arxiv.org/abs/1704.07483&gt;`_.</span>

<span class="sd">    Args:</span>
<span class="sd">        alpha (float): The :math:`\alpha` value for the Celu formulation. Default: 1.0</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - The input of CELU. The required dtype is float16 or float32.</span>
<span class="sd">          The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, with the same type and shape as the `x`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `alpha` is not a float.</span>
<span class="sd">        ValueError: If `alpha` has the value of 0.</span>
<span class="sd">        TypeError: If `x` is not a Tensor.</span>
<span class="sd">        TypeError: If the dtype of &#39;input_x&#39; is neither float16 nor float32.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.array([-2.0, -1.0, 1.0, 2.0]), mindspore.float32)</span>
<span class="sd">        &gt;&gt;&gt; celu = nn.CELU()</span>
<span class="sd">        &gt;&gt;&gt; output = celu(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [-0.86466473 -0.63212055  1.          2.        ]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">1.0</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize CELU.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">CELU</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">celu</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">CeLU</span><span class="p">(</span><span class="n">alpha</span><span class="o">=</span><span class="n">alpha</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">celu</span><span class="p">(</span><span class="n">x</span><span class="p">)</span></div>


<span class="k">class</span> <span class="nc">Softmax</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Softmax activation function.</span>

<span class="sd">    Applies the Softmax function to an n-dimensional input Tensor.</span>

<span class="sd">    The input is a Tensor of logits transformed with exponential function and then</span>
<span class="sd">    normalized to lie in range [0, 1] and sum up to 1.</span>

<span class="sd">    Softmax is defined as:</span>

<span class="sd">    .. math::</span>
<span class="sd">        \text{softmax}(x_{i}) =  \frac{\exp(x_i)}{\sum_{j=0}^{n-1}\exp(x_j)},</span>

<span class="sd">    where :math:`x_{i}` is the :math:`i`-th slice in the given dimension of the input Tensor.</span>

<span class="sd">    Args:</span>
<span class="sd">        axis (Union[int, tuple[int]]): The axis to apply Softmax operation, -1 means the last dimension. Default: -1.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - The input of Softmax with data type of float16 or float32.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, which has the same type and shape as `x` with values in the range[0,1].</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `axis` is neither an int nor a tuple.</span>
<span class="sd">        TypeError: If dtype of `x` is neither float16 nor float32.</span>
<span class="sd">        ValueError: If `axis` is a tuple whose length is less than 1.</span>
<span class="sd">        ValueError: If `axis` is a tuple whose elements are not all in range [-len(x), len(x)).</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)</span>
<span class="sd">        &gt;&gt;&gt; softmax = nn.Softmax()</span>
<span class="sd">        &gt;&gt;&gt; output = softmax(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [0.03168 0.01166 0.0861  0.636   0.2341 ]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">axis</span><span class="o">=-</span><span class="mi">1</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize Softmax.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">Softmax</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">softmax</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Softmax</span><span class="p">(</span><span class="n">axis</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">softmax</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">LogSoftmax</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    LogSoftmax activation function.</span>

<span class="sd">    Applies the LogSoftmax function to n-dimensional input tensor.</span>

<span class="sd">    The input is transformed by the Softmax function and then by the log function to lie in range[-inf,0).</span>

<span class="sd">    Logsoftmax is defined as:</span>

<span class="sd">    .. math::</span>

<span class="sd">        \text{logsoftmax}(x_i) = \log \left(\frac{\exp(x_i)}{\sum_{j=0}^{n-1} \exp(x_j)}\right),</span>

<span class="sd">    where :math:`x_{i}` is the :math:`i`-th slice in the given dimension of the input Tensor.</span>

<span class="sd">    Args:</span>
<span class="sd">        axis (int): The axis to apply LogSoftmax operation, -1 means the last dimension. Default: -1.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - The input of LogSoftmax, with float16 or float32 data type.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, which has the same type and shape as the input as `x` with values in the range[-inf,0).</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `axis` is not an int.</span>
<span class="sd">        TypeError: If dtype of `x` is neither float16 nor float32.</span>
<span class="sd">        ValueError: If `axis` is not in range [-len(x), len(x)).</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32)</span>
<span class="sd">        &gt;&gt;&gt; log_softmax = nn.LogSoftmax()</span>
<span class="sd">        &gt;&gt;&gt; output = log_softmax(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [[-5.00672150e+00 -6.72150636e-03 -1.20067215e+01]</span>
<span class="sd">         [-7.00091219e+00 -1.40009127e+01 -9.12250078e-04]]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">axis</span><span class="o">=-</span><span class="mi">1</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize LogSoftmax.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">LogSoftmax</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">log_softmax</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">LogSoftmax</span><span class="p">(</span><span class="n">axis</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">log_softmax</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">ELU</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Exponential Linear Unit activation function.</span>

<span class="sd">    Applies the exponential linear unit function element-wise.</span>
<span class="sd">    The activation function is defined as:</span>

<span class="sd">    .. math::</span>
<span class="sd">        E_{i} =</span>
<span class="sd">        \begin{cases}</span>
<span class="sd">        x, &amp;\text{if } x \geq 0; \cr</span>
<span class="sd">        \text{alpha} * (\exp(x_i) - 1), &amp;\text{otherwise.}</span>
<span class="sd">        \end{cases}</span>

<span class="sd">    The picture about ELU looks like this `ELU &lt;https://en.wikipedia.org/wiki/</span>
<span class="sd">    Activation_function#/media/File:Activation_elu.svg&gt;`_.</span>

<span class="sd">    Args:</span>
<span class="sd">        alpha (float): The coefficient of negative factor whose type is float. Default: 1.0.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - The input of ELU with data type of float16 or float32.</span>
<span class="sd">          The shape is :math:`(N,*)` where :math:`*` means,any number of additional dimensions.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, with the same type and shape as the `x`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `alpha` is not a float.</span>
<span class="sd">        TypeError: If dtype of `x` is neither float16 nor float32.</span>
<span class="sd">        ValueError: If `alpha` is not equal to 1.0.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float32)</span>
<span class="sd">        &gt;&gt;&gt; elu = nn.ELU()</span>
<span class="sd">        &gt;&gt;&gt; result = elu(x)</span>
<span class="sd">        &gt;&gt;&gt; print(result)</span>
<span class="sd">        [-0.63212055  -0.86466473  0.  2.  1.]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">1.0</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize ELU.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">ELU</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">elu</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Elu</span><span class="p">(</span><span class="n">alpha</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">elu</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">ReLU</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Rectified Linear Unit activation function.</span>

<span class="sd">    Applies the rectified linear unit function element-wise.</span>

<span class="sd">    .. math::</span>

<span class="sd">        \text{ReLU}(x) = (x)^+ = \max(0, x),</span>

<span class="sd">    It returns element-wise :math:`\max(0, x)`, specially, the neurons with the negative output</span>
<span class="sd">    will be suppressed and the active neurons will stay the same.</span>

<span class="sd">    The picture about ReLU looks like this `ReLU &lt;https://en.wikipedia.org/wiki/</span>
<span class="sd">    Activation_function#/media/File:Activation_rectified_linear.svg&gt;`_.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - The input of ReLU. The data type is Number.</span>
<span class="sd">          The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, with the same type and shape as the `x`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If dtype of `x` is not a number.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.array([-1, 2, -3, 2, -1]), mindspore.float16)</span>
<span class="sd">        &gt;&gt;&gt; relu = nn.ReLU()</span>
<span class="sd">        &gt;&gt;&gt; output = relu(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [0. 2. 0. 2. 0.]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize ReLU.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">ReLU</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">relu</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">ReLU</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">relu</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">ReLU6</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Compute ReLU6 activation function.</span>

<span class="sd">    ReLU6 is similar to ReLU with a upper limit of 6, which if the inputs are greater than 6, the outputs</span>
<span class="sd">    will be suppressed to 6.</span>
<span class="sd">    It computes element-wise as</span>

<span class="sd">    .. math::</span>

<span class="sd">        \min(\max(0, x), 6).</span>

<span class="sd">    The input is a Tensor of any valid shape.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - The input of ReLU6 with data type of float16 or float32.</span>
<span class="sd">          The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, which has the same type as `x`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If dtype of `x` is neither float16 nor float32.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)</span>
<span class="sd">        &gt;&gt;&gt; relu6 = nn.ReLU6()</span>
<span class="sd">        &gt;&gt;&gt; output = relu6(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [0. 0. 0. 2. 1.]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize ReLU6.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">ReLU6</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">relu6</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">ReLU6</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">relu6</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">LeakyReLU</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Leaky ReLU activation function.</span>

<span class="sd">    LeakyReLU is similar to ReLU, but LeakyReLU has a slope that makes it not equal to 0 at x &lt; 0.</span>
<span class="sd">    The activation function is defined as:</span>

<span class="sd">    .. math::</span>
<span class="sd">            \text{leaky_relu}(x) = \begin{cases}x, &amp;\text{if } x \geq 0; \cr</span>
<span class="sd">            \text{alpha} * x, &amp;\text{otherwise.}\end{cases}</span>

<span class="sd">    See https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf</span>

<span class="sd">    Args:</span>
<span class="sd">        alpha (Union[int, float]): Slope of the activation function at x &lt; 0. Default: 0.2.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - The input of LeakyReLU.</span>
<span class="sd">          The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, has the same type and shape as the `x`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `alpha` is not a float or an int.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32)</span>
<span class="sd">        &gt;&gt;&gt; leaky_relu = nn.LeakyReLU()</span>
<span class="sd">        &gt;&gt;&gt; output = leaky_relu(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [[-0.2  4.  -1.6]</span>
<span class="sd">         [ 2.  -1.   9. ]]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.2</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize LeakyReLU.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">LeakyReLU</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="n">validator</span><span class="o">.</span><span class="n">check_value_type</span><span class="p">(</span><span class="s1">&#39;alpha&#39;</span><span class="p">,</span> <span class="n">alpha</span><span class="p">,</span> <span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">greater_equal</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">GreaterEqual</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">mul</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Mul</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">alpha</span> <span class="o">=</span> <span class="n">alpha</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">select_op</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Maximum</span><span class="p">()</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">alpha</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">select_op</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Minimum</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="n">alpha_array</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Cast</span><span class="p">()(</span><span class="n">F</span><span class="o">.</span><span class="n">scalar_to_array</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">alpha</span><span class="p">),</span> <span class="n">P</span><span class="o">.</span><span class="n">DType</span><span class="p">()(</span><span class="n">x</span><span class="p">))</span>
        <span class="n">out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">select_op</span><span class="p">(</span><span class="n">alpha_array</span> <span class="o">*</span> <span class="n">x</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">out</span>


<span class="k">class</span> <span class="nc">Tanh</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Tanh activation function.</span>

<span class="sd">    Applies the Tanh function element-wise, returns a new tensor with the hyperbolic tangent of the elements of input,</span>
<span class="sd">    The input is a Tensor with any valid shape.</span>

<span class="sd">    Tanh function is defined as:</span>

<span class="sd">    .. math::</span>
<span class="sd">        tanh(x_i) = \frac{\exp(x_i) - \exp(-x_i)}{\exp(x_i) + \exp(-x_i)} = \frac{\exp(2x_i) - 1}{\exp(2x_i) + 1},</span>

<span class="sd">    where :math:`x_i` is an element of the input Tensor.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - The input of Tanh with data type of float16 or float32.</span>
<span class="sd">          The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, with the same type and shape as the `x`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If dtype of `x` is neither float16 nor float32.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.array([1, 2, 3, 2, 1]), mindspore.float16)</span>
<span class="sd">        &gt;&gt;&gt; tanh = nn.Tanh()</span>
<span class="sd">        &gt;&gt;&gt; output = tanh(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [0.7617 0.964  0.995  0.964  0.7617]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize Tanh.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">Tanh</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">tanh</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Tanh</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">tanh</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">GELU</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Gaussian error linear unit activation function.</span>

<span class="sd">    Applies GELU function to each element of the input. The input is a Tensor with any valid shape.</span>

<span class="sd">    GELU is defined as:</span>

<span class="sd">    .. math::</span>

<span class="sd">        GELU(x_i) = x_i*P(X &lt; x_i),</span>

<span class="sd">    where :math:`P` is the cumulative distribution function</span>
<span class="sd">    of standard Gaussian distribution and :math:`x_i` is the element of the input.</span>

<span class="sd">    The picture about GELU looks like this `GELU &lt;https://en.wikipedia.org/wiki/</span>
<span class="sd">    Activation_function#/media/File:Activation_gelu.png&gt;`_.</span>

<span class="sd">    Args:</span>
<span class="sd">        approximate (bool): Whether to enable approximation. Default: True.</span>

<span class="sd">            If approximate is True, The gaussian error linear activation is:</span>

<span class="sd">            :math:`0.5 * x * (1 + tanh(sqrt(2 / pi) * (x + 0.044715 * x^3)))`</span>

<span class="sd">            else, it is:</span>

<span class="sd">            :math:`x * P(X &lt;= x) = 0.5 * x * (1 + erf(x / sqrt(2)))`, where P(X) ~ N(0, 1).</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - The input of GELU with data type of float16 or float32.</span>
<span class="sd">          The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, with the same type and shape as the `x`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If dtype of `x` is neither float16 nor float32.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32)</span>
<span class="sd">        &gt;&gt;&gt; gelu = nn.GELU()</span>
<span class="sd">        &gt;&gt;&gt; output = gelu(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [[-1.5880802e-01  3.9999299e+00 -3.1077917e-21]</span>
<span class="sd">         [ 1.9545976e+00 -2.2918017e-07  9.0000000e+00]]</span>
<span class="sd">        &gt;&gt;&gt; gelu = nn.GELU(approximate=False)</span>
<span class="sd">        &gt;&gt;&gt; # CPU not support &quot;approximate=False&quot;, using &quot;approximate=True&quot; instead</span>
<span class="sd">        &gt;&gt;&gt; output = gelu(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [[-1.5865526e-01  3.9998732e+00 -0.0000000e+00]</span>
<span class="sd">         [ 1.9544997e+00 -1.4901161e-06  9.0000000e+00]]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">approximate</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize GELU.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">GELU</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="n">validator</span><span class="o">.</span><span class="n">check_bool</span><span class="p">(</span><span class="n">approximate</span><span class="p">,</span> <span class="s1">&#39;approximate&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">approximate</span> <span class="o">=</span> <span class="n">approximate</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">approximate</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">gelu</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">GeLU</span><span class="p">()</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">erf</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Erf</span><span class="p">()</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">sqrt</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Sqrt</span><span class="p">()</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">const0</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">(</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">mstype</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">const1</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">(</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">mstype</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">const2</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">(</span><span class="mf">2.0</span><span class="p">,</span> <span class="n">mstype</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">approximate</span><span class="p">:</span>
            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">gelu</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">x</span> <span class="o">*</span> <span class="n">F</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">const0</span><span class="p">,</span> <span class="n">x</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span> <span class="o">*</span> <span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">const1</span><span class="p">,</span> <span class="n">x</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span> <span class="o">+</span> \
            <span class="bp">self</span><span class="o">.</span><span class="n">erf</span><span class="p">(</span><span class="n">x</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">const2</span><span class="p">,</span> <span class="n">x</span><span class="o">.</span><span class="n">dtype</span><span class="p">))))</span>


<span class="k">class</span> <span class="nc">FastGelu</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Fast Gaussian error linear unit activation function.</span>

<span class="sd">    Applies FastGelu function to each element of the input. The input is a Tensor with any valid shape.</span>

<span class="sd">    FastGelu is defined as:</span>

<span class="sd">    .. math::</span>
<span class="sd">        FastGelu(x_i) = \frac {x_i} {1 + \exp(-1.702 * \left| x_i \right|)} *</span>
<span class="sd">                           \exp(0.851 * (x_i - \left| x_i \right|))</span>

<span class="sd">    where :math:`x_i` is the element of the input.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - The input of FastGelu with data type of float16 or float32.</span>
<span class="sd">          The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, with the same type and shape as the `x`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If dtype of `x` is neither float16 nor float32.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; import mindspore</span>
<span class="sd">        &gt;&gt;&gt; from mindspore import Tensor, nn</span>
<span class="sd">        &gt;&gt;&gt; import numpy as np</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32)</span>
<span class="sd">        &gt;&gt;&gt; fast_gelu = nn.FastGelu()</span>
<span class="sd">        &gt;&gt;&gt; output = fast_gelu(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [[-1.5418735e-01  3.9921875e+00 -9.7473649e-06]</span>
<span class="sd">         [ 1.9375000e+00 -1.0052517e-03  8.9824219e+00]]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize FastGelu.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">FastGelu</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">fast_gelu</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">FastGeLU</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fast_gelu</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">Sigmoid</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Sigmoid activation function.</span>

<span class="sd">    Applies sigmoid-type activation element-wise.</span>

<span class="sd">    Sigmoid function is defined as:</span>

<span class="sd">    .. math::</span>

<span class="sd">        \text{sigmoid}(x_i) = \frac{1}{1 + \exp(-x_i)},</span>

<span class="sd">    where :math:`x_i` is the element of the input.</span>

<span class="sd">    The picture about Sigmoid looks like this `Sigmoid &lt;https://en.wikipedia.org/wiki/</span>
<span class="sd">    Sigmoid_function#/media/File:Logistic-curve.svg&gt;`_.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - The input of Sigmoid with data type of float16 or float32.</span>
<span class="sd">          The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, with the same type and shape as the `x`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If dtype of `x` is neither float16 nor float32.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)</span>
<span class="sd">        &gt;&gt;&gt; sigmoid = nn.Sigmoid()</span>
<span class="sd">        &gt;&gt;&gt; output = sigmoid(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [0.2688  0.11914 0.5     0.881   0.7305 ]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize Sigmoid.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">Sigmoid</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">sigmoid</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Sigmoid</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">sigmoid</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">PReLU</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    PReLU activation function.</span>

<span class="sd">    Applies the PReLU function element-wise.</span>

<span class="sd">    PReLU is defined as:</span>

<span class="sd">    .. math::</span>

<span class="sd">        prelu(x_i)= \max(0, x_i) + w * \min(0, x_i),</span>

<span class="sd">    where :math:`x_i` is an element of an channel of the input.</span>

<span class="sd">    Here :math:`w` is a learnable parameter with a default initial value 0.25.</span>
<span class="sd">    Parameter :math:`w` has dimensionality of the argument channel. If called without argument</span>
<span class="sd">    channel, a single parameter :math:`w` will be shared across all channels.</span>

<span class="sd">    The picture about PReLU looks like this `PReLU &lt;https://en.wikipedia.org/wiki/</span>
<span class="sd">    Activation_function#/media/File:Activation_prelu.svg&gt;`_.</span>

<span class="sd">    Args:</span>
<span class="sd">        channel (int): The elements number of parameter.</span>
<span class="sd">          It could be an int, and the value is 1 or the channels number of input tensor `x`. Default: 1.</span>
<span class="sd">        w (Union[float, list, Tensor]): The initial value of parameter. It could be a float, a float list or</span>
<span class="sd">          a tensor has the same dtype as the input tensor `x`. Default: 0.25.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - The input of PReLU with data type of float16 or float32.</span>
<span class="sd">          The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, with the same dtype and shape as the `x`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `channel` is not an int.</span>
<span class="sd">        TypeError: If `w` is not one of a float, a float list, a float Tensor.</span>
<span class="sd">        TypeError: If dtype of `x` is neither float16 nor float32.</span>
<span class="sd">        ValueError: If the `x` is a 0-D or 1-D Tensor on Ascend.</span>
<span class="sd">        ValueError: If `channel` is less than 1.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.array([[[[0.1, 0.6], [0.9, 0.9]]]]), mindspore.float32)</span>
<span class="sd">        &gt;&gt;&gt; prelu = nn.PReLU()</span>
<span class="sd">        &gt;&gt;&gt; output = prelu(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [[[[0.1 0.6]</span>
<span class="sd">           [0.9 0.9]]]]</span>

<span class="sd">    &quot;&quot;&quot;</span>
    <span class="nd">@cell_attr_register</span><span class="p">(</span><span class="n">attrs</span><span class="o">=</span><span class="s2">&quot;&quot;</span><span class="p">)</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">channel</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">w</span><span class="o">=</span><span class="mf">0.25</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize PReLU.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">PReLU</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="n">validator</span><span class="o">.</span><span class="n">check_positive_int</span><span class="p">(</span><span class="n">channel</span><span class="p">,</span> <span class="s1">&#39;channel&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">w</span><span class="p">,</span> <span class="p">(</span><span class="nb">float</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)):</span>
            <span class="n">tmp</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">empty</span><span class="p">((</span><span class="n">channel</span><span class="p">,),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
            <span class="n">tmp</span><span class="o">.</span><span class="n">fill</span><span class="p">(</span><span class="n">w</span><span class="p">)</span>
            <span class="n">w</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">(</span><span class="n">tmp</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">mstype</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">w</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">w</span><span class="p">)</span> <span class="o">!=</span> <span class="n">channel</span><span class="p">:</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, the length of &#39;w&#39; should be equal to the &#39;channel&#39; when &quot;</span>
                                 <span class="sa">f</span><span class="s2">&quot;the &#39;w&#39; is a list, but got the length of &#39;w&#39;: </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">w</span><span class="p">)</span><span class="si">}</span><span class="s2">, the &#39;channel&#39;: </span><span class="si">{</span><span class="n">channel</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>

            <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">w</span><span class="p">:</span>
                <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">i</span><span class="p">,</span> <span class="p">(</span><span class="nb">float</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)):</span>
                    <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, all elements in &#39;w&#39; should be &quot;</span>
                                     <span class="sa">f</span><span class="s2">&quot;float when the &#39;w&#39; is a list, but got </span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
            <span class="n">w</span> <span class="o">=</span> <span class="n">Tensor</span><span class="p">(</span><span class="n">w</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">mstype</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">w</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">):</span>
            <span class="k">if</span> <span class="n">w</span><span class="o">.</span><span class="n">dtype</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="n">mstype</span><span class="o">.</span><span class="n">float16</span><span class="p">,</span> <span class="n">mstype</span><span class="o">.</span><span class="n">float32</span><span class="p">):</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, the dtype of &#39;w&#39; should be float16 or &quot;</span>
                                 <span class="sa">f</span><span class="s2">&quot;float32 when the &#39;w&#39; is a tensor, but got </span><span class="si">{</span><span class="n">w</span><span class="o">.</span><span class="n">dtype</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">w</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">1</span> <span class="ow">or</span> <span class="n">w</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="n">channel</span><span class="p">:</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, the dimension of &#39;w&#39; should be 1, and the elements number &quot;</span>
                                 <span class="sa">f</span><span class="s2">&quot;should be equal to the &#39;channel&#39; when the &#39;w&#39; is a tensor, &quot;</span>
                                 <span class="sa">f</span><span class="s2">&quot;but got &#39;w&#39; shape </span><span class="si">{</span><span class="n">w</span><span class="o">.</span><span class="n">shape</span><span class="si">}</span><span class="s2">, the &#39;channel&#39; </span><span class="si">{</span><span class="n">channel</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, the &#39;w&#39; only supported float, list and tensor, &quot;</span>
                            <span class="sa">f</span><span class="s2">&quot;but got </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">w</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">w</span> <span class="o">=</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">w</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="s1">&#39;a&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">prelu</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">PReLU</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">relu</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">ReLU</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">assign</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Assign</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="n">u</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">relu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">w</span><span class="p">)</span>
        <span class="n">v</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">prelu</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">F</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">u</span><span class="p">,</span> <span class="n">x</span><span class="o">.</span><span class="n">dtype</span><span class="p">))</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">training</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">assign</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">w</span><span class="p">,</span> <span class="n">u</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">v</span>


<span class="k">class</span> <span class="nc">HSwish</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Hard swish activation function.</span>

<span class="sd">    Applies hswish-type activation element-wise. The input is a Tensor with any valid shape.</span>

<span class="sd">    Hard swish is defined as:</span>

<span class="sd">    .. math::</span>
<span class="sd">        \text{hswish}(x_{i}) = x_{i} * \frac{ReLU6(x_{i} + 3)}{6},</span>

<span class="sd">    where :math:`x_{i}` is the :math:`i`-th slice in the given dimension of the input Tensor.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - The input of HSwish, data type must be float16 or float32.</span>
<span class="sd">          The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, with the same type and shape as the `x`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If dtype of `x` is neither float16 nor float32.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)</span>
<span class="sd">        &gt;&gt;&gt; hswish = nn.HSwish()</span>
<span class="sd">        &gt;&gt;&gt; result = hswish(x)</span>
<span class="sd">        &gt;&gt;&gt; print(result)</span>
<span class="sd">        [-0.3333 -0.3333  0.      1.667   0.6665]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize HSwish.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">HSwish</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">hswish</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">HSwish</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hswish</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">HSigmoid</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Hard sigmoid activation function.</span>

<span class="sd">    Applies hard sigmoid activation element-wise. The input is a Tensor with any valid shape.</span>

<span class="sd">    Hard sigmoid is defined as:</span>

<span class="sd">    .. math::</span>
<span class="sd">        \text{hsigmoid}(x_{i}) = max(0, min(1, \frac{x_{i} + 3}{6})),</span>

<span class="sd">    where :math:`x_{i}` is the :math:`i`-th slice in the given dimension of the input Tensor.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **input_x** (Tensor) - The input of HSigmoid. The shape is :math:`(N,*)` where :math:`*` means, any number of</span>
<span class="sd">          additional dimensions.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, with the same type and shape as the `input_x`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `input_x` is not a Tensor.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)</span>
<span class="sd">        &gt;&gt;&gt; hsigmoid = nn.HSigmoid()</span>
<span class="sd">        &gt;&gt;&gt; result = hsigmoid(x)</span>
<span class="sd">        &gt;&gt;&gt; print(result)</span>
<span class="sd">        [0.3333 0.1666 0.5    0.8335 0.6665]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize HSigmoid.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">HSigmoid</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">hsigmoid</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">HSigmoid</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">input_x</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hsigmoid</span><span class="p">(</span><span class="n">input_x</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">LogSigmoid</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Logsigmoid activation function.</span>

<span class="sd">    Applies logsigmoid activation element-wise. The input is a Tensor with any valid shape.</span>

<span class="sd">    Logsigmoid is defined as:</span>

<span class="sd">    .. math::</span>
<span class="sd">        \text{logsigmoid}(x_{i}) = log(\frac{1}{1 + \exp(-x_i)}),</span>

<span class="sd">    where :math:`x_{i}` is the element of the input.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - The input of LogSigmoid with data type of float16 or float32.</span>
<span class="sd">          The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, with the same type and shape as the `x`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If dtype of `x` is neither float16 nor float32.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; net = nn.LogSigmoid()</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.array([1.0, 2.0, 3.0]), mindspore.float32)</span>
<span class="sd">        &gt;&gt;&gt; output = net(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [-0.31326166 -0.12692806 -0.04858734]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize LogSigmoid.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">LogSigmoid</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">mul</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Mul</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">exp</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Exp</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">add</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Add</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">rec</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Reciprocal</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">log</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Log</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">input_x</span><span class="p">):</span>
        <span class="n">neg_input</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">mul</span><span class="p">(</span><span class="n">input_x</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span>
        <span class="n">exp_neg_input</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="n">neg_input</span><span class="p">)</span>
        <span class="n">exp_neg_input_1</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">exp_neg_input</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
        <span class="n">rec_exp_neg_input_1</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">rec</span><span class="p">(</span><span class="n">exp_neg_input_1</span><span class="p">)</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="n">rec_exp_neg_input_1</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span>


<div class="viewcode-block" id="SoftShrink"><a class="viewcode-back" href="../../../../api_python/nn/mindspore.nn.SoftShrink.html#mindspore.nn.SoftShrink">[docs]</a><span class="k">class</span> <span class="nc">SoftShrink</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Applies the SoftShrink function element-wise.</span>

<span class="sd">    .. math::</span>
<span class="sd">        \text{SoftShrink}(x) =</span>
<span class="sd">        \begin{cases}</span>
<span class="sd">        x - \lambda, &amp; \text{ if } x &gt; \lambda \\</span>
<span class="sd">        x + \lambda, &amp; \text{ if } x &lt; -\lambda \\</span>
<span class="sd">        0, &amp; \text{ otherwise }</span>
<span class="sd">        \end{cases}</span>

<span class="sd">    Args:</span>
<span class="sd">        lambd: the :math:`\lambda` must be no less than zero for the SoftShrink formulation. Default: 0.5.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **input_x** (Tensor) - The input of SoftShrink with data type of float16 or float32.</span>
<span class="sd">          Any number of additional dimensions.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, has the same shape and data type as `input_x`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If lambd is not a float.</span>
<span class="sd">        TypeError: If input_x is not a Tensor.</span>
<span class="sd">        TypeError: If dtype of input_x is neither float16 nor float32.</span>
<span class="sd">        ValueError: If lambd is less than 0.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; input_x = Tensor(np.array([[ 0.5297,  0.7871,  1.1754], [ 0.7836,  0.6218, -1.1542]]), mstype.float16)</span>
<span class="sd">        &gt;&gt;&gt; softshrink = nn.SoftShrink()</span>
<span class="sd">        &gt;&gt;&gt; output = softshrink(input_x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [[ 0.02979  0.287    0.676  ]</span>
<span class="sd">         [ 0.2837   0.1216  -0.6543 ]]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">lambd</span><span class="o">=</span><span class="mf">0.5</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">SoftShrink</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">softshrink</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">SoftShrink</span><span class="p">(</span><span class="n">lambd</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">input_x</span><span class="p">):</span>
        <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">softshrink</span><span class="p">(</span><span class="n">input_x</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">output</span></div>


<span class="k">class</span> <span class="nc">HShrink</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Applies the hard shrinkage function element-wise, each element complies the follow function:</span>

<span class="sd">    .. math::</span>
<span class="sd">        \text{HardShrink}(x) =</span>
<span class="sd">        \begin{cases}</span>
<span class="sd">        x, &amp; \text{ if } x &gt; \lambda \\</span>
<span class="sd">        x, &amp; \text{ if } x &lt; -\lambda \\</span>
<span class="sd">        0, &amp; \text{ otherwise }</span>
<span class="sd">        \end{cases}</span>

<span class="sd">    Args:</span>
<span class="sd">        lambd (float): The value for the HardShrink formulation. Default: 0.5</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **input_x** (Tensor) - The input of HardShrink with data type of float16 or float32.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, the same shape and data type as the input.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend``</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `lambd` is not a float.</span>
<span class="sd">        TypeError: If dtype of `input_x` is neither float16 nor float32.</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; import mindspore</span>
<span class="sd">        &gt;&gt;&gt; from mindspore import Tensor, nn</span>
<span class="sd">        &gt;&gt;&gt; import numpy as np</span>
<span class="sd">        &gt;&gt;&gt; input_x = Tensor(np.array([[ 0.5,  1,  2.0],[0.0533,0.0776,-2.1233]]),mindspore.float32)</span>
<span class="sd">        &gt;&gt;&gt; hshrink = nn.HShrink()</span>
<span class="sd">        &gt;&gt;&gt; output = hshrink(input_x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [[ 0.      1.      2.    ]</span>
<span class="sd">        [ 0.      0.     -2.1233]]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">lambd</span><span class="o">=</span><span class="mf">0.5</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">HShrink</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">hshrink</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">HShrink</span><span class="p">(</span><span class="n">lambd</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">input_x</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hshrink</span><span class="p">(</span><span class="n">input_x</span><span class="p">)</span>


<span class="n">_activation</span> <span class="o">=</span> <span class="p">{</span>
    <span class="s1">&#39;softmax&#39;</span><span class="p">:</span> <span class="n">Softmax</span><span class="p">,</span>
    <span class="s1">&#39;logsoftmax&#39;</span><span class="p">:</span> <span class="n">LogSoftmax</span><span class="p">,</span>
    <span class="s1">&#39;relu&#39;</span><span class="p">:</span> <span class="n">ReLU</span><span class="p">,</span>
    <span class="s1">&#39;relu6&#39;</span><span class="p">:</span> <span class="n">ReLU6</span><span class="p">,</span>
    <span class="s1">&#39;tanh&#39;</span><span class="p">:</span> <span class="n">Tanh</span><span class="p">,</span>
    <span class="s1">&#39;gelu&#39;</span><span class="p">:</span> <span class="n">GELU</span><span class="p">,</span>
    <span class="s1">&#39;fast_gelu&#39;</span><span class="p">:</span> <span class="n">FastGelu</span><span class="p">,</span>
    <span class="s1">&#39;elu&#39;</span><span class="p">:</span> <span class="n">ELU</span><span class="p">,</span>
    <span class="s1">&#39;sigmoid&#39;</span><span class="p">:</span> <span class="n">Sigmoid</span><span class="p">,</span>
    <span class="s1">&#39;prelu&#39;</span><span class="p">:</span> <span class="n">PReLU</span><span class="p">,</span>
    <span class="s1">&#39;leakyrelu&#39;</span><span class="p">:</span> <span class="n">LeakyReLU</span><span class="p">,</span>
    <span class="s1">&#39;hswish&#39;</span><span class="p">:</span> <span class="n">HSwish</span><span class="p">,</span>
    <span class="s1">&#39;hsigmoid&#39;</span><span class="p">:</span> <span class="n">HSigmoid</span><span class="p">,</span>
    <span class="s1">&#39;logsigmoid&#39;</span><span class="p">:</span> <span class="n">LogSigmoid</span><span class="p">,</span>
    <span class="s1">&#39;softshrink&#39;</span><span class="p">:</span> <span class="n">SoftShrink</span><span class="p">,</span>
    <span class="s1">&#39;hshrink&#39;</span><span class="p">:</span> <span class="n">HShrink</span><span class="p">,</span>
<span class="p">}</span>


<span class="k">def</span> <span class="nf">get_activation</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">prim_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Gets the activation function.</span>

<span class="sd">    Args:</span>
<span class="sd">        name (str): The name of the activation function.</span>
<span class="sd">        prim_name (Union[str, None]): The name of primitive. Default: None.</span>

<span class="sd">    Returns:</span>
<span class="sd">        Function, the activation function.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; sigmoid = nn.get_activation(&#39;sigmoid&#39;)</span>
<span class="sd">        &gt;&gt;&gt; print(sigmoid)</span>
<span class="sd">        Sigmoid&lt;&gt;</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">msg_prefix</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="n">prim_name</span><span class="si">}</span><span class="s2">&#39;, the&quot;</span> <span class="k">if</span> <span class="n">prim_name</span> <span class="k">else</span> <span class="s2">&quot;The&quot;</span>
    <span class="k">if</span> <span class="n">name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
        <span class="k">return</span> <span class="kc">None</span>

    <span class="k">if</span> <span class="n">name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">_activation</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">msg_prefix</span><span class="si">}</span><span class="s2"> &#39;name&#39; should be in </span><span class="si">{</span><span class="nb">list</span><span class="p">(</span><span class="n">_activation</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span><span class="si">}</span><span class="s2">, but got </span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">_activation</span><span class="p">[</span><span class="n">name</span><span class="p">]()</span>
</pre></div>

           </div>
           
          </div>
          <footer>

  <hr/>

  <div role="contentinfo">
    <p>
        &#169; Copyright 2021, MindSpore.

    </p>
  </div>
    
    
    
    Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
    
    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
    
    provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>
        </div>
      </div>

    </section>

  </div>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
   

</body>
</html>